
#Importing modules
import cv2
import matplotlib.pyplot as plt
from matplotlib import cm
import numpy as np
import pandas as pd
import IPython.display as ipd
import librosa
from midiutil import MIDIFile
import random


#Load the image
ori_img = cv2.imread('colors.jpg')
img = cv2.cvtColor(ori_img, cv2.COLOR_BGR2RGB)

#Get shape of image
height, width, depth = img.shape
dpi = plt.rcParams['figure.dpi']
figsize = width / float(dpi), height / float(dpi)

#Plot the image
fig, axs = plt.subplots(1, 2, figsize = figsize)
axs[0].title.set_text('BGR') 
axs[0].imshow(ori_img)
axs[1].title.set_text('RGB') 
axs[1].imshow(img)
plt.show()
print('           Image Properties')
print('Height = ',height, 'Width = ', width)
print('Number of pixels in image = ', height * width)

           Image Properties
Height =  360 Width =  640
Number of pixels in image =  230400


#Need function that reads pixel hue value 
hsv = cv2.cvtColor(ori_img, cv2.COLOR_BGR2HSV)
#Plot the image
fig, axs = plt.subplots(1, 3, figsize = (15,15))
names = ['BGR','RGB','HSV']
imgs  = [ori_img, img, hsv]
i = 0
for elem in imgs:
    axs[i].title.set_text(names[i])
    axs[i].imshow(elem)
    axs[i].grid(False)
    i += 1
plt.show()


i=0 ; j=0
#Initialize array the will contain Hues for every pixel in image
hues = [] 
for i in range(height):
    for j in range(width):
        hue = hsv[i][j][0] #This is the hue value at pixel coordinate (i,j)
        hues.append(hue)


pixels_df = pd.DataFrame(hues, columns=['hues'])
pixels_df


#Define frequencies that make up A-Harmonic Minor Scale
scale_freqs = [220.00, 246.94 ,261.63, 293.66, 329.63, 349.23, 415.30] 
def hue2freq(h,scale_freqs):
    thresholds = [26 , 52 , 78 , 104,  128 , 154 , 180]
    note = scale_freqs[0]
    if (h <= thresholds[0]):
         note = scale_freqs[0]
    elif (h > thresholds[0]) & (h <= thresholds[1]):
        note = scale_freqs[1]
    elif (h > thresholds[1]) & (h <= thresholds[2]):
        note = scale_freqs[2]
    elif (h > thresholds[2]) & (h <= thresholds[3]):
        note = scale_freqs[3]
    elif (h > thresholds[3]) & (h <= thresholds[4]):    
        note = scale_freqs[4]
    elif (h > thresholds[4]) & (h <= thresholds[5]):
        note = scale_freqs[5]
    elif (h > thresholds[5]) & (h <= thresholds[6]):
        note = scale_freqs[6]
    else:
        note = scale_freqs[0]
    
    return note


pixels_df['notes'] = pixels_df.apply(lambda row : hue2freq(row['hues'],scale_freqs), axis = 1)     
pixels_df


frequencies = pixels_df['notes'].to_numpy()


song = np.array([]) 
sr = 22050 # sample rate
T = 0.1    # 0.1 second duration
t = np.linspace(0, T, int(T*sr), endpoint=False) # time variable
#Make a song with numpy array :]
#nPixels = int(len(frequencies))#All pixels in image
nPixels = 60
for i in range(nPixels):  
    val = frequencies[i]
    note  = 0.5*np.sin(2*np.pi*val*t)
    song  = np.concatenate([song, note])
ipd.Audio(song, rate=sr) # load a NumPy array


song = np.array([]) 
octaves = np.array([0.5,1,2])
sr = 22050 # sample rate
T = 0.1    # 0.1 second duration
t = np.linspace(0, T, int(T*sr), endpoint=False) # time variable
#Make a song with numpy array :]
#nPixels = int(len(frequencies))#All pixels in image
nPixels = 60
for i in range(nPixels):
    octave = random.choice(octaves)
    val =  octave * frequencies[i]
    note  = 0.5*np.sin(2*np.pi*val*t)
    song  = np.concatenate([song, note])
ipd.Audio(song, rate=sr) # load a NumPy array


song = np.array([]) 
octaves = np.array([1/2,1,2])
sr = 22050 # sample rate
T = 0.1    # 0.1 second duration
t = np.linspace(0, T, int(T*sr), endpoint=False) # time variable
#Make a song with numpy array :]
#nPixels = int(len(frequencies))#All pixels in image
nPixels = 60
for i in range(nPixels):
    octave = random.choice(octaves)
    val =  octave * random.choice(frequencies)
    note  = 0.5*np.sin(2*np.pi*val*t)
    song  = np.concatenate([song, note])
ipd.Audio(song, rate=sr) # load a NumPy array


def img2music(img, scale = [220.00, 246.94 ,261.63, 293.66, 329.63, 349.23, 415.30],
              sr = 22050, T = 0.1, nPixels = 60, useOctaves = True, randomPixels = False):
    """
    Args:
        img    :     (array) image to process
        scale  :     (array) array containing frequencies to map H values to
        sr     :     (int) sample rate to use for resulting song
        T      :     (int) time in seconds for dutation of each note in song
        nPixels:     (int) how many pixels to use to make song
    Returns:
        song   :     (array) Numpy array of frequencies. Can be played by ipd.Audio(song, rate = sr)
    """
    #Convert image to HSV
    hsv = cv2.cvtColor(ori_img, cv2.COLOR_BGR2HSV)
    
    #Get shape of image
    height, width, depth = ori_img.shape

    i=0 ; j=0 ; k=0
    #Initialize array the will contain Hues for every pixel in image
    hues = [] 
    for i in range(height):
        for j in range(width):
            hue = hsv[i][j][0] #This is the hue value at pixel coordinate (i,j)
            hues.append(hue)
            
    #Make dataframe containing hues and frequencies
    pixels_df = pd.DataFrame(hues, columns=['hues'])
    pixels_df['frequencies'] = pixels_df.apply(lambda row : hue2freq(row['hues'],scale), axis = 1) 
    frequencies = pixels_df['frequencies'].to_numpy()

    song = np.array([]) 
    octaves = np.array([0.5,1,2])#Go an octave below, same note, or go an octave above
    t = np.linspace(0, T, int(T*sr), endpoint=False) # time variable
    #Make a song with numpy array :]
    #nPixels = int(len(frequencies))#All pixels in image
    for k in range(nPixels):
        if useOctaves:
            octave = random.choice(octaves)
        else:
            octave = 1
        
        if randomPixels == False:
            val =  octave * frequencies[k]
        else:
            val = octave * random.choice(frequencies)
            
        note  = 0.5*np.sin(2*np.pi*val*t)
        song  = np.concatenate([song, note])
    
    return song


def get_piano_notes():   
    # White keys are in Uppercase and black keys (sharps) are in lowercase
    octave = ['C', 'c', 'D', 'd', 'E', 'F', 'f', 'G', 'g', 'A', 'a', 'B'] 
    base_freq = 440 #Frequency of Note A4
    keys = np.array([x+str(y) for y in range(0,9) for x in octave])
    # Trim to standard 88 keys
    start = np.where(keys == 'A0')[0][0]
    end = np.where(keys == 'C8')[0][0]
    keys = keys[start:end+1]
    
    note_freqs = dict(zip(keys, [2**((n+1-49)/12)*base_freq for n in range(len(keys))]))
    note_freqs[''] = 0.0 # stop
    return note_freqs

def get_sine_wave(frequency, duration, sample_rate=44100, amplitude=4096):
    t = np.linspace(0, duration, int(sample_rate*duration)) # Time axis
    wave = amplitude*np.sin(2*np.pi*frequency*t)
    return wave


def makeScale(whichOctave, whichKey, whichScale):
    
    #Load note dictionary
    note_freqs = get_piano_notes()
    
    #Define tones. Upper case are white keys in piano. Lower case are black keys
    scale_intervals = ['A','a','B','C','c','D','d','E','F','f','G','g']
    
    #Find index of desired key
    index = scale_intervals.index(whichKey)
    
    #Redefine scale interval so that scale intervals begins with whichKey
    new_scale = scale_intervals[index:12] + scale_intervals[:index]
    
    #Choose scale
    if whichScale == 'AEOLIAN':
        scale = [0, 2, 3, 5, 7, 8, 10]
    elif whichScale == 'BLUES':
        scale = [0, 2, 3, 4, 5, 7, 9, 10, 11]
    elif whichScale == 'CHROMATIC':
        scale = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
    elif whichScale == 'DIATONIC_MINOR':
        scale = [0, 2, 3, 5, 7, 8, 10]
    elif whichScale == 'DORIAN':
        scale = [0, 2, 3, 5, 7, 9, 10]
    elif whichScale == 'HARMONIC_MINOR':
        scale = [0, 2, 3, 5, 7, 8, 11]
    elif whichScale == 'LYDIAN':
        scale = [0, 2, 4, 6, 7, 9, 11]
    elif whichScale == 'MAJOR':
        scale = [0, 2, 4, 5, 7, 9, 11]
    elif whichScale == 'MELODIC_MINOR':
        scale = [0, 2, 3, 5, 7, 8, 9, 10, 11]
    elif whichScale == 'MINOR':    
        scale = [0, 2, 3, 5, 7, 8, 10]
    elif whichScale == 'MIXOLYDIAN':     
        scale = [0, 2, 4, 5, 7, 9, 10]
    elif whichScale == 'NATURAL_MINOR':   
        scale = [0, 2, 3, 5, 7, 8, 10]
    elif whichScale == 'PENTATONIC':    
        scale = [0, 2, 4, 7, 9]
    else:
        print('Invalid scale name')
    
    #Get length of scale (i.e., how many notes in scale)
    nNotes = len(scale)
    freqs = []
    for i in range(nNotes):
        note = new_scale[scale[i]] + str(whichOctave)
        freqToAdd = note_freqs[note]
        freqs.append(freqToAdd)
    
    return freqs


#Pixel Art
pixel_art = cv2.imread('pixel_art1.png')
pixel_art2 = cv2.cvtColor(pixel_art, cv2.COLOR_BGR2RGB)
plt.figure()
plt.imshow(pixel_art2)
plt.grid(False)
plt.show()


pixel_scale = makeScale(4, 'a', 'HARMONIC_MINOR')
pixel_song  = img2music(pixel_art, pixel_scale, T = 0.2, randomPixels = True)
ipd.Audio(pixel_song, rate = sr)


#Waterfall
waterfall = cv2.imread('waterfall.jpg')
waterfall2 = cv2.cvtColor(waterfall, cv2.COLOR_BGR2RGB)
plt.figure()
plt.imshow(waterfall2)
plt.grid(False)
plt.show()


waterfall_scale = makeScale(1, 'd', 'MAJOR')
waterfall_song  = img2music(waterfall, waterfall_scale, T = 0.3, randomPixels = True, useOctaves = True)
ipd.Audio(waterfall_song, rate = sr)


#Peacock
peacock = cv2.imread('peacock.jpg')
peacock2 = cv2.cvtColor(peacock, cv2.COLOR_BGR2RGB)
plt.figure()
plt.imshow(peacock2)
plt.grid(False)
plt.show()


peacock_scale = makeScale(3, 'E', 'DORIAN')
peacock_song  = img2music(peacock, peacock_scale, T = 0.2, randomPixels = False,
                          useOctaves = True, nPixels = 120)
ipd.Audio(peacock_song, rate = sr)


#Cat
cat = cv2.imread('cat1.jpg')
cat2 = cv2.cvtColor(cat, cv2.COLOR_BGR2RGB)
plt.figure()
plt.imshow(cat2)
plt.grid(False)
plt.show()


cat_scale = makeScale(2, 'f', 'AEOLIAN')
cat_song  = img2music(cat, cat_scale, T = 0.4, randomPixels = True,
                          useOctaves = True, nPixels = 120)
ipd.Audio(cat_song, rate = sr)


#water
water = cv2.imread('water.jpg')
water2 = cv2.cvtColor(water, cv2.COLOR_BGR2RGB)
plt.figure()
plt.imshow(water2)
plt.grid(False)
plt.show()


water_scale = makeScale(2, 'B', 'LYDIAN')
water_song  = img2music(water, water_scale, T = 0.2, randomPixels = False,
                          useOctaves = True, nPixels = 60)
ipd.Audio(water_song, rate = sr)


#earth
earth = cv2.imread('earth.jpg')
earth2 = cv2.cvtColor(earth, cv2.COLOR_BGR2RGB)
plt.figure()
plt.imshow(earth2)
plt.grid(False)
plt.show()


earth_scale = makeScale(3, 'g', 'MELODIC_MINOR')
earth_song  = img2music(earth, earth_scale, T = 0.3, randomPixels = False,
                          useOctaves = True, nPixels = 60)
ipd.Audio(earth_song, rate = sr)


midi_vals  = [37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49]
def freq2midi(freq, midi_vals):
    midi_freqs = [219, 233 ,247, 262, 277, 294, 311, 330, 349, 370, 392, 392, 415.3] 
    note = scale_freqs[0]
    if (freq <= midi_freqs[0]):
         midi_number = midi_vals[0]
    elif (freq > midi_freqs[0]) & (freq <= midi_freqs[1]):
        midi_number = midi_vals[1]
    elif (freq > midi_freqs[1]) & (freq <= midi_freqs[2]):
        midi_number = midi_vals[2]
    elif (freq > midi_freqs[2]) & (freq <= midi_freqs[3]):
        midi_number = midi_vals[3]
    elif (freq > midi_freqs[3]) & (freq <= midi_freqs[4]):    
        midi_number = midi_vals[4]
    elif (freq > midi_freqs[4]) & (freq <= midi_freqs[5]):
        midi_number = midi_vals[5]
    elif (freq > midi_freqs[6]) & (freq <= midi_freqs[6]):
        midi_number = midi_vals[6]
    elif (freq > midi_freqs[7]) & (freq <= midi_freqs[7]):    
        midi_number = midi_vals[7]
    elif (freq > midi_freqs[8]) & (freq <= midi_freqs[8]):
        midi_number = midi_vals[8]
    elif (freq > midi_freqs[9]) & (freq <= midi_freqs[9]):
        midi_number = midi_vals[9]
    elif (freq > midi_freqs[10]) & (freq <= midi_freqs[10]):
        midi_number = midi_vals[10]
    elif (freq > midi_freqs[11]) & (freq <= midi_freqs[11]):
        midi_number = midi_vals[11]
    else:
        midi_number = midi_vals[0]
    
    return midi_number


hues_df['midi_number'] = hues_df.apply(lambda row : freq2midi(row['notes'],midi_vals), axis = 1) 
hues_df


midi_number = hues_df['midi_number'].to_numpy()


degrees  = list(midi_number) # MIDI note number
track    = 0
channel  = 0
time     = 0   # In beats
duration = 1   # In beats
tempo    = 240  # In BPM
volume   = 100 # 0-127, as per the MIDI standard

MyMIDI = MIDIFile(1) # One track, defaults to format 1 (tempo track
                     # automatically created)
MyMIDI.addTempo(track,time, tempo)

for pitch in degrees:
    MyMIDI.addNote(track, channel, pitch, time, duration, volume)
    time = time + 1
with open("song.mid", "wb") as output_file:
    MyMIDI.writeFile(output_file)


#Convert frequency to a note
hues_df['notes2'] = hues_df.apply(lambda row : librosa.hz_to_note(row['notes']), axis = 1)  
#Convert note to a midi number
hues_df['midi_number2'] = hues_df.apply(lambda row : librosa.note_to_midi(row['notes2']), axis = 1)    
hues_df


#Convert midi number column to a numpy array
midi_number = hues_df['midi_number2'].to_numpy()


degrees  = list(midi_number) # MIDI note number
track    = 0
channel  = 0
time     = 0   # In beats
duration = 1   # In beats
tempo    = 240  # In BPM
volume   = 100 # 0-127, as per the MIDI standard

MyMIDI = MIDIFile(1) # One track, defaults to format 1 (tempo track
                     # automatically created)
MyMIDI.addTempo(track,time, tempo)

for pitch in degrees:
    MyMIDI.addNote(track, channel, pitch, time, duration, volume)
    time = time + 1
with open("image2music_1.mid", "wb") as output_file:
    MyMIDI.writeFile(output_file)


#Convert hue to wavelength[nm] via interpolation. Assume spectrum is contained between 400-650nm
def hue2wl(h, wlMax = 650, wlMin = 400, hMax = 270, hMin = 0):
    #h *= 2
    hMax /= 2
    hMin /= 2
    wlRange = wlMax - wlMin
    hRange = hMax - hMin
    wl =  wlMax - ((h* (wlRange))/(hRange))
    return wl


#Array with hue values from 0 degrees to 270 degrees
h_array = np.arange(0,270,1)
h_array.shape

# define vectorized sigmoid
hue2wl_v = np.vectorize(hue2wl)
test = hue2wl_v(h_array)
test.shape
np.min(test)

plt.title("Interpolation of Hue and Wavelength") 
plt.xlabel("Hue()") 
plt.ylabel("Wavelength[nm]") 
plt.scatter(h_array, test, c = cm.gist_rainbow_r(np.abs(h_array)), edgecolor='none')
plt.gca().invert_yaxis()
plt.style.use('seaborn-darkgrid')
plt.show()


#Convert hue to wavelength
hues_df['nm'] = hues_df.apply(lambda row : hue2wl(row['hues']), axis = 1)    
hues_df['nm'].min()


img = cv2.imread('colors.jpg')
#Convert a hue value to wavelength via interpolation
#Assume that visible spectrum is contained between 400-650nm
def hue2wl(h, wlMax = 650, wlMin = 400, hMax = 270, hMin = 0):
    #h *= 2
    hMax /= 2
    hMin /= 2
    wlRange = wlMax - wlMin
    hRange = hMax - hMin
    wl =  wlMax - ((h* (wlRange))/(hRange))
    return wl

def wl2freq(wl):
    wavelength = wl
    sol = 299792458.00 #this is the speed of light in m/s
    sol *= 1e9 #Convert speed of light to nm/s
    freq = (sol / wavelength) * (1e-12)
    return freq

def img2music(img, fName):
    
    #Get height and width of image
    height, width, _ = img.shape
    
    #Convet from BGR to HSV
    hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
    
    #Populate hues array with H channel for each pixel
    i=0 ; j=0
    hues = []
    for i in range(height):
        for j in range(width):
            hue = hsv[i][j][0] #This is the hue value at pixel coordinate (i,j)
            hues.append(hue)
            
    #Make pandas dataframe        
    hues_df = pd.DataFrame(hues, columns=['hues'])
    hues_df['nm'] = hues_df.apply(lambda row : hue2wl(row['hues']), axis = 1)  
    hues_df['freq'] = hues_df.apply(lambda row : wl2freq(row['nm']), axis = 1) 
    hues_df['notes'] = hues_df.apply(lambda row : librosa.hz_to_note(row['freq']), axis = 1)  
    hues_df['midi_number'] = hues_df.apply(lambda row : librosa.note_to_midi(row['notes']), axis = 1) 
    
    #Convert midi number column to a numpy array
    midi_number = hues_df['midi_number'].to_numpy()
    
    #Enter MIDI parameters
    degrees  = list(midi_number) # MIDI note number
    track    = 0
    channel  = 0
    time     = 0   # In beats
    duration = 1   # In beats
    tempo    = 240  # In BPM
    volume   = 100 # 0-127, as per the MIDI standard
    
    #Make MIDI file
    #MyMIDI = MIDIFile(1) # One track (tempo track automatically created)
    #MyMIDI.addTempo(track, time, tempo)
    #for pitch in degrees:
    #    MyMIDI.addNote(track, channel, pitch, time, duration, volume)
    #    time = time + 1
    #with open(fName + ".mid", "wb") as output_file:
    #   MyMIDI.writeFile(output_file)
    
    #midiName = fName + ".mid"
    print("Done making song from image!") 
    
    return hues_df


df = img2music(img,'color')
df

Done making song from image!


#Convert midi number column to a numpy array
sr = 22050 # sample rate
song = df['freq'].to_numpy()
ipd.Audio(song, rate = sr) # load a NumPy array


a_HarmonicMinor = [220.00, 246.94 ,261.63, 293.66, 329.63, 349.23, 415.30, 440.00] 
frequencies = df['freq'].to_numpy()
song = np.array([]) 
harmony = np.array([]) 
octaves = np.array([1/4,1,2,1,2])
sr = 22050 # sample rate
T = 0.25    # 0.1 second duration
t = np.linspace(0, T, int(T*sr), endpoint=False) # time variable
#Make a song with numpy array :]
nPixels = int(len(frequencies)/height)
nPixels = 30
#for j in tqdm(range(nPixels), desc="Processing Frame"):#Add progress bar for frames processed      
for i in range(nPixels):  
    octave = random.choice(octaves)
    val =  octave * frequencies[i]
    note  = 0.5*np.sin(2*np.pi*val*t)
    song  = np.concatenate([song, note])
ipd.Audio(song, rate=sr) # load a NumPy array

	hues
0	8
1	31
2	21
3	26
4	33
...	...
230395	20
230396	20
230397	17
230398	17
230399	22

Making Music From Images

Modules Used in this work

Loading Image

Using HSV Color Space

Extract Hue from Image

Converting hues to frequencies (1st Idea)

Turning frequencies into MIDI values

Using Librosa For Mapping

Converting hues to frequencies (2nd Idea)

	hues	notes
0	8	220.00
1	31	246.94
2	21	220.00
3	26	220.00
4	33	246.94
...	...	...
230395	20	220.00
230396	20	220.00
230397	17	220.00
230398	17	220.00
230399	22	220.00